/* Generate all IRi-based side artifacts (descriptives etc)
	cdo  "F:\MonPol Project\ReplicationPackage\Dofiles\a3_SideAnalyses.do"
 */

* Technicalities
** Parameters
global sample "_10pct" 	// _10pct _10pctB or _sample

global graphSizeFreqDensity ""
global graphSizeFreqDensity_res "1"


global rootL "F:\MonPol Project\ReplicationPackage\Large"
global rootD "F:\MonPol Project\ReplicationPackage"

** Programs
qui do "$rootD/Dofiles/999_Paths"
qui do "$rootD/Dofiles/998_Programs"

cap log close log
log using "$path_logs/a0_ImportData${sample}.smcl", append smcl name(log)

* Code
**# graphSizeFreqDensity
if "$graphSizeFreqDensity" == "1" {
    if !inlist("$sample", "", "_sample") error 999
	
	* Start log
    cap log close graphSizeFreqDensity
    log using "$path_logs/graphSizeFreqDensity.log", append text name(graphSizeFreqDensity)
	cap mkdir "$path_graphs/deviationGraphs${sample}"
	cap mkdir "$path_graphs/deviationGraphs${sample}/comp_new"
	cap mkdir "$path_graphs/deviationGraphs${sample}/hybrid_new"
	cap mkdir "$path_graphs/deviationGraphs${sample}/reset_new"
	
	* Competitor prices
	use month id_nr_unique PRF L_xCompDem_PRF using "$path_regressionData/allCompetitorPrices_PRF${sample}", replace
	gen DL_PRF = D.PRF 
	gen byte D_PRF = abs(DL_PRF) > 0.0001 & DL_PRF != .	
	rename (DL_PRF D_PRF)(DL_PRF_h1 D01_PRF_h1)
	local nBins = 31 
	sum L_xCompDem_PRF, d
	global binwidth = int(round(100*(r(p99)-r(p1))/`nBins'))
	dummyGenerator L_xCompDem_PRF, gapcount(7) middlebinsize(10) binname(gapDummy_comp) show graph p(PRF) graphbinwidth(${binwidth}) gaplabel("Competitor price gap (in %)") xlabel(-50(10)50) path("$path_graphs/deviationGraphs${sample}/comp_new")  edge(0.5) noxlines		// Figure 4: The size and frequency of subsequent reference-price changes as a function of the competitor-price gap and its density [Top Row]
	
	
	* Competitor-reset prices
	use id_nr month D_PRF DL_PRF count_PRF count_DPRF L_xHybridDem_DPRF using "$path_regressionData/allHybridPrices_DPRF${sample}", replace
	drop if count_DPRF < 50
	drop if missing(L_xHybridDem_DPRF)
	rename (DL_PRF D_PRF)(DL_PRF_h1 D01_PRF_h1)
	
	local nBins = 31 
	sum L_xHybridDem_DPRF, d
	global binwidth = int(round(100*(r(p99)-r(p1))/`nBins'))
	dummyGenerator L_xHybridDem_DPRF, gapcount(7) middlebinsize(10) binname(gapDummy_hybrid) show graph p(PRF) graphbinwidth(${binwidth}) gaplabel("Competitor-reset price gap (in %)") xlabel(-50(10)50) path("$path_graphs/deviationGraphs${sample}/hybrid_new")  edge(0.5) noxlines		// Figure 14: The size and the frequency of subsequent reference-price changes as a function of the competitorreset-price-gap and its density [Top Row]
	
	* Reset prices
	use id_nr month D_PRF DL_PRF L_x_PRF using "$path_regressionData/allResetPrices_PRF${sample}", replace
	drop if missing(L_x_PRF )
	rename (DL_PRF D_PRF)(DL_PRF_h1 D01_PRF_h1)
	
	local nBins = 31 
	sum L_x_PRF, d
	global binwidth = int(round(100*(r(p99)-r(p1))/`nBins'))
	dummyGenerator L_x_PRF, gapcount(7) middlebinsize(10) binname(gapDummy_hybrid) show graph p(PRF) graphbinwidth(${binwidth}) gaplabel("Reset price gap (in %)") xlabel(-50(10)50) path("$path_graphs/deviationGraphs${sample}/reset_new")  edge(0.5) noxlines		// Figure 15: Reset-price-gap density and the subsequent frequency and size of price changes as a function of the gap, scanner data + Figure 16: Frequency and size of subsequent price changes as a function of reset-price gap
    
    * Close log
    log close graphSizeFreqDensity
}

**# graphSizeFreqDensity_res		[use _10pct sample, otherwise need insane pc]
if "$graphSizeFreqDensity_res" == "1" {
    * Start log
    cap log close graphSizeFreqDensity_res
    log using "$path_logs/graphSizeFreqDensity_res.log", append text name(graphSizeFreqDensity_res)
	cap mkdir "$path_graphs/deviationGraphs${sample}"
	cap mkdir "$path_graphs/deviationGraphs${sample}/compResidualised"
	cap mkdir "$path_graphs/deviationGraphs${sample}/compResetResidualised"

	/* Competitor prices */
	* Load data
	use id_nr_unique month D01_PRF_h* D_PRF_h* DL_PRF_h* L_xCompDem_PRF ebpnew_residual L_xLnPriceAge category using "$path_data\regressionData/allResetCompPriceAgePrices_PRF${sample}_prepped", replace
	
	gen D01_PRF_h24 = (D_PRF_h24 !=0) if D_PRF_h24 != .
	keep if L_xCompDem_PRF != .
	drop ebpnew_residual category
	compress
	
	
	* Part 1: Residualised gaps (item/time/price age effects filtered out)
	** Residualise
	reghdfe L_xCompDem_PRF L_xLnPriceAge, absorb(id_nr_unique month) residuals(res1)
	sum L_xCompDem_PRF if e(sample), d
	gen gap_comp_residualised = r(mean) + res1
	sum gap_comp_residualised, d
	corr gap_comp_residualised L_xCompDem_PRF
	drop res1 L_xCompDem_PRF
	
	** Discretise
	replace gap_comp_residualised = round(gap_comp_residualised*100, 1)
	replace gap_comp_residualised = 50 if gap_comp_residualised > 50 & gap_comp_residualised != .
	replace gap_comp_residualised = -50 if gap_comp_residualised < -50
	recast byte gap_comp_residualised
	
	
	** Get densities
	*** Count occurences
	frame copy default counts, replace
	frame counts {
		gen one = 1 if !missing(gap_comp_residualised)
		gcollapse (sum) count=one, by(gap_comp_residualised) fast
		drop if missing(gap_comp_residualised)
		
		*** Divide by total
		sum count, meanonly
		gen density = count/r(sum)*100
		
		sum density
		local max = round(r(max), 1)
		twoway (bar density gap_comp_residualised, barwidth(1.2) fcolor(black%50) lcolor(white)), legend(off) graphregion(color(white)) ylabel(, glcolor(black%5)) xlabel(-50(10)50) ylabel(0(1)`max', angle(horizontal)) ytitle("in %") scale(1.4) xtitle("Competitor-price gap (in %)")	// Figure 4: The size and frequency of subsequent reference-price changes as a function of the competitor-price gap and its density [Bottom Row]
		graphsave3 $path_graphs/deviationGraphs${sample}/compResidualised/density
		save "$path_graphs/deviationGraphs${sample}/compResidualised/density", replace
	}
	frame drop counts
		
	* Part 2: Residualised frequencies
	** Make gap groups positive and label (needed for reghdfe, factor variables cannot be negative)
	replace gap_comp_residualised = gap_comp_residualised + 50
	
	label define gapLabel 50 "0", replace
	forvalues i = -50/50 {
		label define gapLabel `=50+`i'' "`i'", modify
	}
	label values gap_comp_residualised gapLabel
	
	** Regress frequency/size on gap groups + item/time FE and price age
	local horizons "1"
	foreach h of local horizons {	
		* Generate up/down indicators
		gen freq_any_h`h'  	= D01_PRF_h`h'
		gen freq_up_h`h' 	= (D_PRF_h`h' == 1)
		gen freq_down_h`h' 	= (D_PRF_h`h' == -1)
		
		gen size_any_h`h'  	= DL_PRF_h`h' 			if abs(DL_PRF_h`h') > 0.01
		gen size_up_h`h'   	= size_any_h`h'		if size_any_h`h' > 0
		gen size_down_h`h' 	= size_any_h`h'		if size_any_h`h' < 0
		
		* Loop over freq/size and any/up/down
		local dvs "freq size"
		local directions "any up down"
		foreach dv of local dvs {
			foreach dir of local directions {
				** Regress on 50x2+1 gap indicators
				reghdfe `dv'_`dir'_h`h' L_xLnPriceAge ib50.gap_comp_residualised, absorb(id_nr_unique month) compact poolsize(30)
				mat B_`dv'_`dir' = e(b)
				
				** Calculate reference value as
				**  Sample average - differential gap group effect
				**  That is, the average filtering out the effect of having a gap that is not zero (the reference)
				gen adjustmentFactor = 0
				di "Allocating differential gap coefficient to observations"
				forvalues i = 0(1)100 {
					di "`i' " _continue
					cap noisily qui replace adjustmentFactor = adjustmentFactor + _b[`i'.gap_comp_residualised]*(gap_comp_residualised==`i')
				}
				di ""
				
				gen dv_withoutGapEffect = `dv'_`dir'_h`h' - adjustmentFactor
				sum `dv'_`dir'_h`h' if gap_comp_residualised == 50, meanonly		// == average at gap 0 [*]
				sum dv_withoutGapEffect
				scalar filteredSampleAverage = r(mean)
				
				** Save and graph gap-specific effects
				*** Store in separate frame
				cap frame drop gapEffects
				frame create gapEffects
				cwf gapEffects
				set obs 101
				gen gap_comp_residualised = .
				gen `dv'_`dir'_h`h'_mean = .
				qui forvalues i = 0(1)100 {
					replace gap_comp_residualised = `i' in `=`i' + 1'
					cap replace `dv'_`dir'_h`h'_mean = _b[`i'.gap_comp_residualised] in `=`i' + 1'
				}
				
				*** Convert gaps back to human-friendly format 
				replace gap_comp_residualised = gap_comp_residualised - 50					// Back to actual deviation
				
				*** Finalise gap effect (add zero-effect and turn into percentage)
				replace `dv'_`dir'_h`h'_mean = `dv'_`dir'_h`h'_mean + scalar(filteredSampleAverage)		// Add average mean at zero gap (=from differential coefficients to mean frequencies)
				replace `dv'_`dir'_h`h'_mean = `dv'_`dir'_h`h'_mean * 100								// to %
				
				*** Determine nice ylabels
				sum `dv'_`dir'_h`h'_mean
				local min = min(r(min), 0)
				local min = `min' - mod(`min', 10)
				di `min'

				local max = r(max)
				local max = `max' - mod(`max', 10) + 10
				di `max'
				
				*** Graph
				twoway (bar `dv'_`dir'_h`h'_mean gap_comp_residualised, barwidth(1.2) fcolor(black%50) lcolor(white)), legend(off) graphregion(color(white)) ylabel(, glcolor(black%5)) xlabel(-50(10)50) ylabel(`min'(10)`max', angle(horizontal)) ytitle("in %") xtitle("Competitor-price gap (in %)") scale(1.4)	// Figure 4: The size and frequency of subsequent reference-price changes as a function of the competitor-price gap and its density [Bottom Row]
				graphsave3 $path_graphs/deviationGraphs${sample}/compResidualised/`dv'_`dir'_h`h'

				*** Save this subset
				save "$path_graphs/deviationGraphs${sample}/compResidualised/`dv'_`dir'_h`h'_mean", replace
				
				** Restore dataset to original form
				cwf default
				drop adjustmentFactor dv_withoutGapEffect
			}
		}	
	}
	
	* Part 3: save combined dataset
	use "$path_graphs/deviationGraphs${sample}/compResidualised/density", replace
	local files: dir "$path_graphs/deviationGraphs${sample}/compResidualised" files "*_mean.dta"
	foreach file of local files {
		merge 1:1 gap_comp_residualised using "$path_graphs/deviationGraphs${sample}/compResidualised/`file'", nogen
	}
	order gap count density freq* size*
	
	save "$path_graphs/deviationGraphs${sample}/compResidualised/dispersionData", replace

	
	
	
	/* Competitor-Reset prices */
	* Load data
	use "$path_regressionData/allResetCompPriceAgePrices_PRF${sample}_prepped", replace
	merge 1:1 id_nr_unique month using "$path_regressionData/allHybridPrices_DPRF${sample}", keep(match master) keepusing(count_DPRF L_xHybridDem_DPRF) nogen
	
	gen D01_PRF_h24 = (D_PRF_h24 !=0) if D_PRF_h24 != .
	drop if count_DPRF < 50
	keep if L_xHybridDem_DPRF != .
	drop ebpnew_residual count_DPRF L_xCompDem_PRF category
	compress
	
	* Part 1: Residualised gaps (item/time/price age effects filtered out)
	** Residualise
	reghdfe L_xHybridDem_DPRF L_xLnPriceAge, absorb(id_nr_unique month) residuals(res1)
	sum L_xHybridDem_DPRF if e(sample), d
	gen gap_compReset_residualised = r(mean) + res1
	sum gap_compReset_residualised, d
	corr gap_compReset_residualised L_xHybridDem_DPRF
	drop res1 L_xHybridDem_DPRF
	
	** Discretise
	replace gap_compReset_residualised = round(gap_compReset_residualised*100, 1)
	replace gap_compReset_residualised = 50 if gap_compReset_residualised > 50 & gap_compReset_residualised != .
	replace gap_compReset_residualised = -50 if gap_compReset_residualised < -50
	recast byte gap_compReset_residualised
	
	
	** Get densities
	*** Count occurences
	frame copy default counts, replace
	frame counts {
		gen one = 1 if !missing(gap_compReset_residualised)
		gcollapse (sum) count=one, by(gap_compReset_residualised) fast
		drop if missing(gap_compReset_residualised)
		
		*** Divide by total
		sum count, meanonly
		gen density = count/r(sum)*100
		twoway (bar density gap_compReset_residualised, barwidth(1.2) fcolor(black%50) lcolor(white)), legend(off) graphregion(color(white)) ylabel(, glcolor(black%5)) xlabel(-50(10)50) ylabel(0(1)5, angle(horizontal)) ytitle("in %") scale(1.4) xtitle("Competitor-reset-price gap (in %)")
		graphsave3 $path_graphs/deviationGraphs${sample}/compResetResidualised/density
		save "$path_graphs/deviationGraphs${sample}/compResetResidualised/density", replace
	}
	frame drop counts
		
	* Part 2: Residualised frequencies
	** Make gap groups positive and label (needed for reghdfe, factor variables cannot be negative)
	replace gap_compReset_residualised = gap_compReset_residualised + 50
	
	label define gapLabel 50 "0", replace
	forvalues i = -50/50 {
		label define gapLabel `=50+`i'' "`i'", modify
	}
	label values gap_compReset_residualised gapLabel
	
	** Regress frequency/size on gap groups + item/time FE and price age
	local horizons "1" 	// [*]
	foreach h of local horizons {	
		* Generate up/down indicators
		gen freq_any_h`h'  	= D01_PRF_h`h'
		gen freq_up_h`h' 	= (D_PRF_h`h' == 1)
		gen freq_down_h`h' 	= (D_PRF_h`h' == -1)
		
		gen size_any_h`h'  	= DL_PRF_h`h' 			if abs(DL_PRF_h`h') > 0.01
		gen size_up_h`h'   	= size_any_h`h'		if size_any_h`h' > 0
		gen size_down_h`h' 	= size_any_h`h'		if size_any_h`h' < 0
		
		* Loop over freq/size and any/up/down
		local dvs "freq size"
		local directions "any up down"
		foreach dv of local dvs {
			foreach dir of local directions {
				** Regress on 50x2+1 gap indicators
				cap noisily reghdfe `dv'_`dir'_h`h' L_xLnPriceAge ib50.gap_compReset_residualised, absorb(id_nr_unique month) compact poolsize(40)
				if _rc == 2001 {
					continue
				} 
				mat B_`dv'_`dir' = e(b)
				
				** Calculate reference value as
				**  Sample average - differential gap group effect
				**  That is, the average filtering out the effect of having a gap that is not zero (the reference)
				gen adjustmentFactor = 0
				di "Allocating differential gap coefficient to observations"
				forvalues i = 0(1)100 {
					di "`i' " _continue
					cap noisily qui replace adjustmentFactor = adjustmentFactor + _b[`i'.gap_compReset_residualised]*(gap_compReset_residualised==`i')
				}
				di ""
				
				gen dv_withoutGapEffect = `dv'_`dir'_h`h' - adjustmentFactor
				sum `dv'_`dir'_h`h' if gap_compReset_residualised == 50		// == average at gap 0
				sum dv_withoutGapEffect
				scalar filteredSampleAverage = r(mean)
				
				** Save and graph gap-specific effects
				*** Store in separate frame
				cap frame drop gapEffects
				frame create gapEffects
				cwf gapEffects
				set obs 101
				gen gap_compReset_residualised = .
				gen `dv'_`dir'_h`h'_mean = .
				qui forvalues i = 0(1)100 {
					replace gap_compReset_residualised = `i' in `=`i' + 1'
					cap replace `dv'_`dir'_h`h'_mean = _b[`i'.gap_compReset_residualised] in `=`i' + 1'
				}
				
				*** Convert gaps back to human-friendly format 
				replace gap_compReset_residualised = gap_compReset_residualised - 50					// Back to actual deviation
				
				*** Finalise gap effect (add zero-effect and turn into percentage)
				replace `dv'_`dir'_h`h'_mean = `dv'_`dir'_h`h'_mean + scalar(filteredSampleAverage)		// Add average mean at zero gap (=from differential coefficients to mean frequencies)
				replace `dv'_`dir'_h`h'_mean = `dv'_`dir'_h`h'_mean * 100								// to %
				
				*** Determine nice ylabels
				sum `dv'_`dir'_h`h'_mean
				local min = min(r(min), 0)
				local min = `min' - mod(`min', 10)
				di `min'

				local max = r(max)
				local max = `max' - mod(`max', 10) + 10
				di `max'
				
				*** Graph
				twoway (bar `dv'_`dir'_h`h'_mean gap_compReset_residualised, barwidth(1.2) fcolor(black%50) lcolor(white)), legend(off) graphregion(color(white)) ylabel(, glcolor(black%5)) xlabel(-50(10)50) ylabel(`min'(10)`max', angle(horizontal)) ytitle("in %") xtitle("Competitor-reset-price gap (in %)") scale(1.4)
				graphsave3 $path_graphs/deviationGraphs${sample}/compResetResidualised/`dv'_`dir'_h`h'

				*** Save this subset
				//merge 1:1 gap_compReset_residualised using "$path_graphs/deviationGraphs${sample}/compResetResidualised/density", keepusing(density) nogen
				save "$path_graphs/deviationGraphs${sample}/compResetResidualised/`dv'_`dir'_h`h'_mean", replace
				
				** Restore dataset to original form
				cwf default
				drop adjustmentFactor dv_withoutGapEffect
			}
		}	
	}
	
	* Part 3: save combined dataset
	use "$path_graphs/deviationGraphs${sample}/compResetResidualised/density", replace
	local files: dir "$path_graphs/deviationGraphs${sample}/compResetResidualised" files "*_mean.dta"
	foreach file of local files {
		merge 1:1 gap_compReset_residualised using "$path_graphs/deviationGraphs${sample}/compResetResidualised/`file'", nogen
	}
	order gap count density freq* size*
	
	save "$path_graphs/deviationGraphs${sample}/compResetResidualised/dispersionData", replace
	

    * Close log
    log close graphSizeFreqDensity_res
}

